package tcp

import (
	"os"
)

//爬虫抓取URL地址
func SpiderDemo() {

	worklist := make(chan []string)
	var n int
	n++
	go func() {
		worklist <- os.Args[1:]
	}()

	//存储已经访问过的URL
	seen := make(map[string]bool)
	for ; n > 0; n-- {
		list := <-worklist
		for _, url := range list {
			if !seen[url] {
				seen[url] = true
				n++
				go func(link string) {
					worklist <- crawl(link)
				}(url)
			}
		}
	}
}

var tokens = make(chan struct{})

func crawl(link string) []string {

	tokens <- struct{}{}
	//list, err := Extract(link)
	//if err != nil {
	//	log.Print(err)
	//}
	return []string{}
}
